#!/usr/bin/env python
#-*- coding:utf-8 -*-

import StringIO
from HTMLParser import HTMLParser, HTMLParseError

class CountParser(HTMLParser):
    structureTags = ('div', 'span', 'ui', 'ul', 'li', 'table', ' br', 'h1', 'h2')
    dummyTags = ('script', 'meta', 'style', 'link')
    def __init__(self):
        HTMLParser.__init__(self)
        self.items = list()
        self.current = CountItem()
        self.flag = True
    def handle_comment(self, data):
        pass

    def handle_starttag(self, tag, attrs):
        if tag in CountParser.structureTags:
            if len(self.current.getText()) > 0:
                self.items.append(self.current)
                self.current = CountItem()
            if self.flag == False:
                self.flag = True
        elif tag in CountParser.dummyTags:
            self.flag = False
        elif tag == 'a':
            self.current.addLink()

    def handle_data(self, data):
        if self.flag:
            self.current.addItem(data)

    def handle_startendtag(self, tag, attrs):
        if tag == 'a':
            self.current.addLink()

    def handle_endtag(self, tag):
        if tag in CountParser.dummyTags:
            self.flag = True
    
    def findPrev(self, start, cutoff):
        last = start
        while start > 0 and start >= 0:
            item = self.items[start]
            slen = len(item.getText())
            if slen > cutoff:
                start -= 1
                last = start
            else:
                start -= 1
                if (last - start) > 4:
                    break
        return last

    def findNext(self, end, cutoff):
        last = end
        while end < len(self.items) and end >= 0:
            item = self.items[end]
            slen = len(item.getText())
            if slen > cutoff:
                end += 1
                last = end
            else:
                end += 1
                if (end - last) > 4:
                    break
        return last

    def getMainContent(self):
        self.items.append(self.current)
        max = 0
        maxIndex = 0
        index = 0
        for i in self.items:
            slen = len(i.getText())
            index += 1
            if slen > max:
                maxIndex = index
                max = slen
        cutoff = float(max) * 0.33
        maxIndex -= 1
        start = maxIndex
        end = maxIndex
        start = self.findPrev(start, cutoff)
        end = self.findNext(end, cutoff)
        res = StringIO.StringIO()
        print max, start, end, maxIndex
        for i in range(start, end + 1):
            item = self.items[i]
            res.write(item.getText())
        return res.getvalue()

class CountItem:
    def __init__(self):
        self.items = 0
        self.string = StringIO.StringIO()
        self.links = 0
    def addLink(self):
        self.links += 1

    def addItem(self, string):
        self.string.write(string)
        self.items += 1

    def getText(self):
        return self.string.getvalue()

if __name__ == "__main__":
    fd = open("data.htm", "r")
    html = fd.read()
    fd.close()
    parser = CountParser()
    parser.feed(html.decode('utf-8'))
    wfd = open("res.data", 'w')
    wfd.write(parser.getMainContent().encode('utf-8'))
    wfd.close()
